import urllib.request, os, sys

class Spider:
    def __init__(self):
        if(os.getcwd().find("blogs")==-1):
            path = "blogs"
            os.chdir(sys.path[0]+"/"+path)
        file = open("list.txt", "a+")
        file.close()

    def request(self,url):
        try:
            file = urllib.request.urlopen(url,timeout=100)
            html = file.read().decode("utf-8","ignore")
            return html
        except Exception as err:
            print(str(err))
            return "error"

    def save(self,list):
        l=self.read()
        if(len(l)>0):
            b=set(l)
            result = list-b
            print("%d %d %d " % (len(b), len(list), len(result)))
        else:
             result = list
        file = open("list.txt", "a")
        for item in result:
            file.write(item)
            file.write("\n")
        file.close()

    def download(self, id,href):
        html = self.request(href)
        if(html!="error"):
            file = open(str(id)+".txt","w")
            file.write(html)
            file.close()

    def read(self):
        file = open("list.txt","r+")
        l = file.readlines()
        file.close()
        for i in range(0,len(l)):
            l[i] = l[i].replace("\n","")
        return l



